package com.gxljc.bear.crawler.util;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.io.StringReader;
import java.io.UnsupportedEncodingException;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.io.IOUtils;
import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpClient;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.params.AllClientPNames;
import org.apache.http.client.params.ClientPNames;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.protocol.BasicHttpContext;
import org.apache.http.protocol.ExecutionContext;
import org.apache.http.protocol.HttpContext;
import org.codehaus.jackson.map.ObjectMapper;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;

import com.mysql.jdbc.StringUtils;
import com.gxljc.commons.util.StringUtil;

import net.sf.json.JSONObject;
import net.sf.json.xml.XMLSerializer;

/**
 * @author tanghaitao
 */
public class HtmlUtil {
	final static String HTTP = "http://";
	final static String HTTPS = "https://";
	final static String[] IMAGE_EXTENSION = new String[] { "jpg", "JPG", "png", "PNG", "jpeg", "JPEG" };
	private final static int RETRY_TIME = 3;
	private static final int SUCCESS_RESPONSE_CODE = 200;

	public static String replaceSpace(String title) {
		if (title == null || title.length() <= 0)
			return title;
		return title.replace("&nbsp;", " ").replace("&amp;", "&").replace("&quot;", "\"").replace("&acute;", "'");
	}

	public static boolean checkImage(String html) {
		if (StringUtils.isNullOrEmpty(html))
			return false;
		if (!html.startsWith(HTTP) && !html.startsWith(HTTPS))
			return false;
		for (String extension : IMAGE_EXTENSION) {
			if (html.endsWith(extension))
				return true;
		}
		return false;
	}

	/**
	 * @param request
	 * @return http请求
	 * @throws Exception
	 * @author haitao
	 */
	public static byte[] fetch(String request, String charset) throws Exception {
		System.out.println(String.format("fetch result for url=%s", request));
		URL url = new URL(request);
		HttpURLConnection conn = (HttpURLConnection) url.openConnection();
		conn.connect();
		int responseCode = conn.getResponseCode();
		if (responseCode != SUCCESS_RESPONSE_CODE) {
			return null;
		}
		InputStream in = conn.getInputStream();
		Reader read = new InputStreamReader(in);
		return IOUtils.toByteArray(read, charset);
	}
	
	public static byte[] fetch(String request) throws Exception{
		return fetch(request,"utf-8");
	}

	// 获取domain
	public static String getHeadHost4URL(String url) {
		final String patternUrl = "http[s]?://(.*)/";
		Pattern p = Pattern.compile(patternUrl);
		Matcher m = p.matcher(url);
		String host = "";
		if (m.find()) {
			host = m.group(1);
		}
		return host;
	}

	/*
	 * HTML Samples: <sup>[1]</sup> <sup>[2-6]</sup>
	 */
	public static String removeNumberIndex(String source) {
		if (StringUtils.isNullOrEmpty(source))
			return source;
		String regEx = "<sup>\\[.*?\\]</sup>";
		Pattern pattern = Pattern.compile(regEx);
		Matcher matcher = pattern.matcher(source);
		String result = matcher.replaceAll("");
		return result;
	}

	public static boolean UrlIsContain(String url, String filter_word) {
		if (url.contains(filter_word))
			return true;
		return false;
	}

	/**
	 * 正则抽取内容
	 *
	 * @param word
	 * @param regex
	 * @return
	 * @date 2022年4月20日 下午2:37:57
	 */
	public static String extractByRegex(String word, String regex) {
		Pattern pattern = Pattern.compile(regex);
		Matcher matcher = pattern.matcher(word);
		StringBuffer strBuff = new StringBuffer();
		while (matcher.find()) {
			strBuff.append(matcher.group());
		}
		return strBuff.toString();
	}

	/**
	 * 判断是否为URL
	 *
	 * @param url
	 * @return
	 */
	public static boolean isUrl(String url) {
		if (url.startsWith(HTTP) || url.startsWith(HTTPS))
			return true;
		return false;
	}

	public static String subStringColumn(String title, int max) {
		if (title == null || title.length() <= 0)
			return title;
		int len = title.length();
		if (len <= max)
			max = len;
		return title.substring(0, max);
	}

	public static String urlEncode(String keyword) {
		try {
			return URLEncoder.encode(keyword, "utf-8");
		} catch (UnsupportedEncodingException e) {
			e.printStackTrace();
		}
		return "";
	}

	public static String pictureUrlPref(String picUrl, String domainUrl) {
		if (picUrl.startsWith(HTTP) || picUrl.startsWith(HTTPS))
			return picUrl;
		if (picUrl.startsWith("/")) {
			String prefUrl = domainUrl.substring(0, domainUrl.indexOf("/", HTTP.length() + 3));
			return prefUrl + picUrl;
		} else if (picUrl.startsWith("../")) {
			final String FLAG = "../";
			domainUrl = domainUrl.substring(0, domainUrl.lastIndexOf("/"));
			while (picUrl.startsWith(FLAG)) {
				domainUrl = domainUrl.substring(0, domainUrl.lastIndexOf("/"));
				picUrl = picUrl.substring(FLAG.length());
			}
			return domainUrl + "/" + picUrl;
		} else if (picUrl.startsWith("./")) {
			final String FLAG = "./";
			domainUrl = domainUrl.substring(0, domainUrl.lastIndexOf("/"));
			picUrl = picUrl.substring(FLAG.length());
			return domainUrl + "/" + picUrl;
		} else {
			String prefUrl = domainUrl.substring(0, domainUrl.lastIndexOf("/") + 1);
			return prefUrl + picUrl;
		}
	}

	@SuppressWarnings("unused")
	public static void main(String[] args) throws Exception {
		String domainUrl = "http://magazine.caijing.com.cn/20160909/4174770.shtml";
		String string = "safsdf      <a charset=\"hz-4006554-1000595\" href=\"http://www.youku.com/show_page/id_z3bca1a9259df11e5a080.html\" target=\"";
	
		System.out.println(new String(fetch(domainUrl)));
	}

	@SuppressWarnings("unchecked")
	public static Map<String, Object> parseJson2Map(String json) {
		Map<String, Object> dataMap = null;
		try {
			dataMap = new ObjectMapper().readValue(new StringReader(json), Map.class);
		} catch (Exception e) {
			e.printStackTrace();
		}
		return dataMap;
	}

	/**
	 * @param picUrl
	 *            图片相对路径
	 * @param crawlUrl
	 *            抓取网页
	 * @return 图片绝对路径
	 *         <p>
	 *         // ../../../images/2016-08/18/01/res17_attpic_brief.jpg //
	 *         http://epaper.bjbusiness.com.cn/site1/bjsb/html/2016-08/18/
	 *         content_344993.htm // ->
	 *         http://epaper.bjbusiness.com.cn/site1/bjsb/images/2016-08/18/01/
	 *         res17_attpic_brief.jpg
	 */
	public static String genAbsolutePicUrl(String picUrl, String crawlUrl) {
		if (picUrl.startsWith(HTTP) || picUrl.startsWith(HTTPS)) {
			return picUrl;
		}
		crawlUrl = crawlUrl.substring(0, crawlUrl.lastIndexOf('/'));
		while (picUrl.startsWith(".")) {
			crawlUrl = crawlUrl.substring(0, crawlUrl.lastIndexOf('/'));
			picUrl = picUrl.substring(picUrl.indexOf('/') + 1);
		}
		return crawlUrl + "/" + picUrl;
	}

	/**
	 * 把list转为String
	 *
	 * @param text
	 * @return
	 */
	@SuppressWarnings("unchecked")
	public static String textToString(Object text, String split) {
		if (text == null)
			return null;
		if (!(text instanceof List))
			return text.toString();
		StringBuffer sb = new StringBuffer();
		List<String> ts = (ArrayList<String>) text;
		for (String t : ts) {
			if (sb.length() > 0)
				sb.append(split);
			sb.append(t);
		}
		return sb.toString();
	}

	public static String genSiteDomain(String url) {
		final String URL_SEPARATOR = "/";
		if (StringUtils.isNullOrEmpty(url))
			return null;
		url = url.replace(HTTP, "");
		url = url.replace(HTTPS, "");
		if (url.indexOf(URL_SEPARATOR) >= 0) {
			return url.substring(0, url.indexOf(URL_SEPARATOR));
		}
		return url;
	}

	public static String http302Page(String url) throws Exception {

		CloseableHttpClient httpClient = HttpClients.createDefault();
		HttpGet httpget = new HttpGet(url);
		HttpContext localContext = new BasicHttpContext();
		httpget.setHeader(AllClientPNames.HANDLE_REDIRECTS, "false");
		httpget.getParams().setParameter(ClientPNames.HANDLE_REDIRECTS, false);
		httpget.setHeader("Referer", url);
		CloseableHttpResponse response = httpClient.execute(httpget, localContext);
		try {
			int status = response.getStatusLine().getStatusCode();
			System.out.println(status);
			org.apache.http.Header[] hs = response.getHeaders("Location");
			for (org.apache.http.Header h : hs) {
				System.out.println(h);
			}
			// System.out.println(response.getHeaders("Location"));;
			HttpHost target = (HttpHost) localContext.getAttribute(ExecutionContext.HTTP_TARGET_HOST);
			HttpUriRequest req = (HttpUriRequest) localContext.getAttribute(ExecutionContext.HTTP_REQUEST);
			return target.toString() + req.getURI();
		} catch (Exception er) {
			er.printStackTrace();
		}
		return "";

		// HttpClient httpClient = new HttpClient();
		// GetMethod getMethod = new GetMethod(url);
		// getMethod.setRequestHeader("http.protocol.handle-redirects","false");
		// getMethod.getParams().setContentCharset("GBK");
		// getMethod.getParams().setParameter(AllClientPNames.HANDLE_REDIRECTS,false);
		// int statusCode = httpClient.executeMethod(getMethod);
		// String to = "";
		// System.out.println(statusCode);
		// if(statusCode == 301 || statusCode == 302){
		// Header[] hs = getMethod.getResponseHeaders("Location");
		// for (Header h : hs) {
		// to = h.getValue();
		// }
		// }
		// httpClient.getHttpConnectionManager().getParams()
		// .setConnectionTimeout(5000);
		// httpClient.getHttpConnectionManager().getParams().setSoTimeout(20000);
		//
		// String to = "";
		// org.apache.commons.httpclient.HttpClient client = new
		// org.apache.commons.httpclient.HttpClient();
		// HttpGet httpget = new HttpGet(url);
		// HttpParams params = client.getParams();
		// params.setParameter(AllClientPNames.HANDLE_REDIRECTS, false);
		// try {
		// HttpResponse response = client.executeMethod(httpget);
		// int statusCode = response.getStatusLine().getStatusCode();
		// if (statusCode == 301 || statusCode == 302) {
		// Header[] hs = response.getHeaders("Location");
		// for (Header h : hs) {
		// to = h.getValue();
		// }
		// }
		// } catch (Exception e) {
		// to = "";
		// }
		// return to;

		// serverUrl = new URL(location);
		// conn = (HttpURLConnection) serverUrl.openConnection();
		// conn.setRequestMethod("GET");
		//
		// conn.addRequestProperty("Accept-Charset", "UTF-8;");
		// conn.addRequestProperty("User-Agent",
		// "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9.2.8)
		// Firefox/3.6.8");
		// conn.addRequestProperty("Referer", "http://javaniu.com/");
		// conn.connect();
		// System.out.println("跳转地址:" + location);
		// return location;
	}

	/**
	 * 删除多余空格
	 *
	 * @param text
	 * @return
	 */
	public static String deleteSpaceString(String text) {
		if (text == null)
			return null;
		text = text.trim().replace(" ", "").replace(" ", "");
		return text;
	}

	/**
	 * 字符串统计
	 *
	 * @param text
	 * @param word
	 * @return
	 */
	public static int countInText(String text, String word) {
		if (StringUtils.isNullOrEmpty(text))
			return 0;
		if (StringUtils.isNullOrEmpty(word))
			return 0;
		int ind = text.indexOf(word);
		int count = 0;
		while (ind >= 0) {
			count++;
			ind = text.indexOf(word, ind + 1);
		}
		return count;
	}

	/**
	 * 抓取网页HttpURLConnection
	 *
	 * @param urlPath
	 * @return
	 * @throws Exception
	 */
	public String getJsonString(String urlPath, String charset) throws Exception {
		URL url = new URL(urlPath);
		HttpURLConnection connection = (HttpURLConnection) url.openConnection();
		connection.connect();
		InputStream inputStream = connection.getInputStream();
		Reader reader = new InputStreamReader(inputStream, charset);
		BufferedReader bufferedReader = new BufferedReader(reader);
		String str = null;
		StringBuffer sb = new StringBuffer();
		while ((str = bufferedReader.readLine()) != null) {
			sb.append(str);
		}
		reader.close();
		connection.disconnect();
		return sb.toString();
	}

	/**
	 * 抓取网页 httpClient
	 *
	 * @param urlPath
	 * @param charset
	 * @return
	 * @throws ClientProtocolException
	 * @throws IOException
	 */
	@SuppressWarnings({})
	public String getHtmlContent(String urlPath, String charset) throws ClientProtocolException, IOException {
		HttpClient httpClient = HttpClients.custom().setMaxConnTotal(400).setMaxConnPerRoute(400)
				.setDefaultRequestConfig(RequestConfig.custom().setSocketTimeout(30000).setConnectTimeout(5000).build())
				.setRetryHandler(new DefaultHttpRequestRetryHandler(5, true)).build();
		HttpGet request = new HttpGet(urlPath);
		HttpResponse response = httpClient.execute(request);
		// 服务请求状态码小于四百的为正常
		if (response.getStatusLine().getStatusCode() < 400) {
			BufferedReader rd = new BufferedReader(new InputStreamReader(response.getEntity().getContent(), charset));
			String line = "";
			StringBuffer sb = new StringBuffer();
			while ((line = rd.readLine()) != null) {
				sb.append(line);
			}
			return sb.toString();
		} else {
			return null;
		}
	}

	/**
	 * 豆瓣screenShot正则匹配
	 */
	public String extractMovieTimeByRegex(String html) {
		Pattern pattern = Pattern.compile("<li id=\"v_0\">.*/li>");
		Matcher matcher = pattern.matcher(html);
		StringBuffer strBuff = new StringBuffer();
		while (matcher.find()) {
			strBuff.append(matcher.group());
		}
		return strBuff.toString();
	}

	public static String xml2JSON(String xml) {
		return new XMLSerializer().read(xml).toString();
	}

	/**
	 * 抓取豆瓣的screenShot
	 *
	 * @param urlPath
	 * @return
	 * @throws Exception
	 */
	public Map<String, Object> crawlerDoubanScreenShot(String urlPath) {
		String content = null;
		Map<String, Object> screenShotMap = new HashMap<>();
		try {
			content = getHtmlContent(urlPath, "utf-8");
		} catch (Exception e) {
			e.printStackTrace();
		}
		try {
			if (!StringUtil.isNullOrEmpty(content)) {
				String pgx = extractMovieTimeByRegex(content);
				String pgxSplit = pgx.split(" </li>")[0];
				String screenshot = extractByRegex(pgxSplit, "<img src=.*/>").split("=")[1].split(" ")[0]
						.replaceAll("\"", "");
				String time = extractByRegex(pgxSplit, "<em>[0-9:：]+</em>").split(">")[1].split("<")[0];
				String title = extractByRegex(pgxSplit, "<strong>.*</strong>").split(">")[1].split("<")[0];
				screenShotMap.put("playDuration", time);
				screenShotMap.put("screenshot", screenshot);
				screenShotMap.put("title", title);
			}
		} catch (Exception e) {
			e.printStackTrace();
		}
		return screenShotMap;
	}

	/**
	 * 抓取爱奇艺screenshot
	 *
	 * @param urlPath
	 * @return
	 */
	@SuppressWarnings({ "static-access", "rawtypes" })
	public Map<String, Object> crawlerIqiYiScreenShot(String urlPath) {
		// 构造url，先取tvId(电影id)，vid(图片id)，之后再访问http://cache.video.qiyi.com/*接口
		String IqiyiUrl = "http://cache.video.qiyi.com/jp/vi/%s/%s/?callback=QiyiPlayerLoader.__callbacks__.iqqhv92l";
		String IqiyiUrlTime = "http://mixer.video.iqiyi.com/jp/mixin/videos/%s?callback=window.Q.__callbacks__.cbmonxmy&status=1";
		String iqiyiScreenShot = null;
		String playDuration = null;
		Map<String, Object> iqiyiScreenDuration = new HashMap<>();
		try {
			String htmlContent = getHtmlContent(urlPath, "utf-8");
			String tvId = htmlContent.split("tvid=")[1].split("\"")[1];
			String vid = htmlContent.split("data-player-videoid=")[1].split("\"")[1];
			String tvIdUrl = IqiyiUrl.format(IqiyiUrl, tvId, vid);
			String tvIdUrlTime = IqiyiUrlTime.format(IqiyiUrlTime, tvId);
			String htmlContentTime = getHtmlContent(tvIdUrlTime, "utf-8");
			htmlContentTime = htmlContentTime.replace("try{window.Q.__callbacks__.cbmonxmy(", "")
					.replace(");}catch(e){};", "");
			JSONObject jsonObjectTime = JSONObject.fromObject(htmlContentTime);
			Map data = (Map) jsonObjectTime.get("data");
			Object duration = data.get("duration");
			if (duration != null) {
				Double durationDouble = new Double((Integer) duration);
				playDuration = handleSoHuPlayDuration(durationDouble);
			}
			String picContent = getHtmlContent(tvIdUrl, "utf-8")
					.replace("try{QiyiPlayerLoader.__callbacks__.iqqhv92l(", "").replace(");}catch(e){};", "");
			JSONObject jsonObject = JSONObject.fromObject(picContent);
			iqiyiScreenShot = jsonObject.get("apic").toString();
			// 添加尺寸116*65
			String[] aqiyiScreenShotSplit = iqiyiScreenShot.split("\\.jpg");
			iqiyiScreenShot = aqiyiScreenShotSplit[0] + "_116_65.jpg";
			iqiyiScreenDuration.put("playDuration", playDuration);
			iqiyiScreenDuration.put("screenshot", iqiyiScreenShot);
		} catch (Exception e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}

		return iqiyiScreenDuration;
	}

	/**
	 * 抓取搜狐的screenshot
	 *
	 * @param urlPath
	 * @return
	 */
	@SuppressWarnings("rawtypes")
	public Map<String, Object> crawlerSohuScreenShot(String urlPath) {
		// 构造url，先取tvId(电影id)，vid(图片id)，之后再访问http://cache.video.qiyi.com/*接口
		String sohuScreenShot = null;
		String sohuPlayDuration = null;
		String urlFormat = "http://pl.hd.sohu.com/videolist?playlistid=%s&o_playlistId=&vid=%s&pianhua=0&pageRule=undefined&pagesize=999&order=0&cnt=1&callback=__get_videolist";
		Map<String, Object> sohuScreenDuration = new HashMap<>();
		try {
			// 搜狐的网页有些电影没有PlayDuration，或者ScreenShot。又或者两者都有
			String content = getHtmlContent(urlPath, "gbk");
			String[] urlSplit = content.split("http://hotvrs.js.tv.itc.cn/");
			if (content.contains("videoLength:")) {
				try {
					String playDucation = content.split("videoLength:")[1].split("\'")[1];
					sohuPlayDuration = handleSoHuPlayDuration(Double.parseDouble(playDucation));
				} catch (Exception e) {
					// TODO: handle exception
				}
			} else if (content.contains("playlistId=")) {
				String playListId = content.split("playlistId=")[1].split("\"")[1];
				String vid = extractByRegex(content, "vid=.*var nid =").split("vid=")[1].split("\"")[1];
				String htmlUrl = String.format(urlFormat, playListId, vid);
				String htmlContent = getHtmlContent(htmlUrl, "gbk").replaceAll("__get_videolist\\(|\\);", "");
				JSONObject jsonObject = JSONObject.fromObject(htmlContent);
				List videos = (List) jsonObject.get("videos");
				Map video = (Map) videos.get(0);
				double playLength = (double) video.get("playLength");
				sohuPlayDuration = handleSoHuPlayDuration(playLength);
			}
			String urlSecond = urlSplit[1].split("\"")[0];
			String picJsUrl = "http://hotvrs.js.tv.itc.cn/" + urlSecond;
			String contentJs = getHtmlContent(picJsUrl, "utf-8");
			contentJs = contentJs.replaceAll("init_video_info\\(|\\);", "");
			JSONObject jsonObject = JSONObject.fromObject(contentJs);
			sohuScreenShot = jsonObject.get("pic170110").toString();
		} catch (Exception e) {
			e.printStackTrace();
		}
		sohuScreenDuration.put("playDuration", sohuPlayDuration);
		sohuScreenDuration.put("screenshot", sohuScreenShot);
		return sohuScreenDuration;
	}

	/**
	 * 搜狐视频playDuration转化，给出的是546这样的数字，需要取模求余来计算分钟，秒:02:36
	 *
	 * @param playDuration
	 * @return
	 */
	String handleSoHuPlayDuration(Double playDuration) {
		Double fPlayDuration = playDuration;
		double hourFloat = fPlayDuration / 60;
		int hourInt = (int) hourFloat;
		String hourString = String.valueOf(hourInt);
		double ddAfterDot = hourFloat - hourInt;
		if (hourInt < 10) {
			hourString = "0" + hourInt;
		}
		long minute = Math.round(ddAfterDot * 60);
		String minuteStr = String.valueOf(minute);
		if (minute < 10) {
			minuteStr = "0" + minuteStr;
		}
		return hourString + ":" + minuteStr;
	}

	/**
	 * 抓取优酷的screenshot
	 *
	 * @param urlPath
	 * @return
	 */
	public Map<String, Object> crawlerYouKuScreenShot(String urlPath) {
		Map<String, Object> YouKuScreenShot = new HashMap<>();
		String playUrlHtml = null;
		String playDuration = null;
		if (urlPath.contains("keyword=")) {
			try {
				String content = getHtmlContent(urlPath, "UTF-8");
				String playUrl = content.split("iku_showid=")[1].split("href=")[1].split("\"")[1];
				if (!playUrl.contains("http")) {
					playUrl = "http://www.soku.com" + playUrl;
				}
				playUrlHtml = getHtmlContent(playUrl, "UTF-8");
				playDuration = YouKuPlayDuracation(playUrlHtml);
				String playUrlScreenShot = playUrlHtml.split("&pics=")[1].split("&site=")[0];
				YouKuScreenShot.put("url", playUrl);
				YouKuScreenShot.put("screenshot", playUrlScreenShot);
				YouKuScreenShot.put("playDuration", playDuration == null ? "" : playDuration);
			} catch (Exception e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
		} else {
			try {
				playUrlHtml = getHtmlContent(urlPath, "UTF-8");
				playDuration = YouKuPlayDuracation(playUrlHtml);
				String playUrlScreenShot = playUrlHtml.split("&pics=")[1].split("&site=")[0];
				YouKuScreenShot.put("url", urlPath);
				YouKuScreenShot.put("screenshot", playUrlScreenShot);
				YouKuScreenShot.put("playDuration", playDuration == null ? "" : playDuration);
			} catch (Exception e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
		}
		return YouKuScreenShot;
	}

	public String YouKuPlayDuracation(String playUrlHtml) {
		String match = extractByRegex(playUrlHtml, "<a charset=.*</span>");
		String playDuracation = match.split("href=")[1].split("\"")[1];
		String playDuracationHtml;
		String playDuracationYK = null;
		try {
			playDuracationHtml = getHtmlContent(playDuracation, "utf-8");
			playDuracationYK = playDuracationHtml.split("时长:</label>")[1].split("<")[0].replaceAll("\\s*", "");

		} catch (ClientProtocolException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		return playDuracationYK;
	}

	/**
	 * 抓取mgtv的screenshot
	 *
	 * @param urlPath
	 * @return
	 */
	@SuppressWarnings({ "static-access", "rawtypes" })
	public Map<String, Object> crawlerMgTvScreenShot(String urlPath) {
		String urlFormat = "http://v.api.mgtv.com/list/movielist?video_id=%s&callback=jQuery182010534219583496451_1468762287313&_=1468762287421";
		Map<String, Object> mgTvScreenDuration = new HashMap<>();
		String mgTvScreenShot = null;
		String mgTvPlayDuration = null;
		try {
			String content = getJsonString(urlPath, "UTF-8");
			String playCid = content.split("player_cid: ")[1].split(",")[0];
			String picUrl = urlFormat.format(urlFormat, playCid);
			String picHtml = getJsonString(picUrl, "UTF-8");
			picHtml = picHtml.replaceAll("jQuery182010534219583496451_1468762287313\\(|\\)", "");
			JSONObject jsonObject = JSONObject.fromObject(picHtml);
			Map data = (Map) jsonObject.get("data");
			Map info = (Map) data.get("info");
			if (info != null) {
				mgTvScreenShot = info.get("img").toString();
				mgTvPlayDuration = info.get("duration").toString();
				mgTvScreenDuration.put("screenshot", mgTvScreenShot);
				mgTvScreenDuration.put("playDuration", mgTvPlayDuration);
			}
		} catch (Exception e) {
			e.printStackTrace();
		}
		return mgTvScreenDuration;
	}

	/**
	 * 抓取网页
	 *
	 * @param request
	 * @return
	 * @throws Exception
	 */
	public String getHtmlByUrl(String request) throws Exception {
		int retry = 0;
		while (retry < RETRY_TIME) {
			URL url = new URL(request);
			HttpURLConnection httpURL = (HttpURLConnection) url.openConnection();
			httpURL.setConnectTimeout(2000);
			httpURL.setReadTimeout(3000);
			httpURL.connect();
			InputStream input = httpURL.getInputStream();
			StringBuffer sb = new StringBuffer("");
			byte[] b = new byte[1024];
			while (input.read(b) != -1) {
				sb.append(new String(b, "utf-8"));
			}
			if (!StringUtil.isNullOrEmpty(sb.toString())) {
				return sb.toString();
			}
			retry++;
		}
		return null;
	}

	// 匹配出网页中包含的图片
	public static List<String> getImageSrc(String htmlCode) {
		List<String> imageSrcList = new ArrayList<String>();
		Pattern p = Pattern.compile(
				"<img\\b[^>]*\\bsrc\\b\\s*=\\s*('|\")?([^'\"\n\r\f>]+(\\.jpg|\\.bmp|\\.eps|\\.gif|\\.mif|\\.miff|\\.png|\\.tif|\\.tiff|\\.svg|\\.wmf|\\.jpe|\\.jpeg|\\.dib|\\.ico|\\.tga|\\.cut|\\.pic)\\b)[^>]*>",
				Pattern.CASE_INSENSITIVE);
		Matcher m = p.matcher(htmlCode);
		String quote = null;
		String src = null;
		while (m.find()) {
			quote = m.group(1);
			src = (quote == null || quote.trim().length() == 0) ? m.group(2).split("\\s+")[0] : m.group(2);
			imageSrcList.add(src);
		}
		return imageSrcList;
	}

	@SuppressWarnings("unused")
	public static void getJsoupHtml() {
		try {
			Document doc = Jsoup.connect("http://www.iqiyi.com/v_19rrkv5504.html").get();
		} catch (Exception e) {
			e.printStackTrace();
		}
	}

}
